#!/usr/bin/python
# -*- coding: cp1250 -*-

import string
import pepe.glib as g
import shutil, glob
import re

import pepe.p_utils.srch as s
import pepe.p_utils.srchr as sr
import pepe.p_utils.csv_cntdel as cd
import pepe.p_txt.bigtxt as bt
from pepe.glib import findcoldelimiter_line

from pepe.utils import u

#for inpfile in glob.glob('Deal_Header_UTF-8_Russia_Legal.get-sample.#.csv'):
for inpfile in glob.glob('*.txt'):

    fname = string.split(inpfile, '.')[0]

    o = open(fname + '_repaired', 'w')
    r = open(fname + '_repair_log', 'w')

    buffer_line = ''

    lino = 0
    len_cols = 0

    cd = ''

    bigest_line = 0
    biggest_column = 0

    for line in bt.lines(inpfile):
    
        if len(line) > bigest_line: bigest_line = len(line)

        if cd == '': cd = findcoldelimiter_line(line)
    
        lino += 1
        line = re.sub('\n', '', line)
        # remove tabs in pipe delimited text file
        if cd <> '\t': line = re.sub('\t', ' ', line)
        
        cols = string.split(line, cd)
        for c in cols:
            if len(c) > biggest_column: biggest_column = len(c)

        #print len(cols)
        # take header len cols as reference
        if len_cols == 0: len_cols = len(cols)

        if len(cols) == len_cols:
            o.write(line + '\n')
        else:
            if buffer_line == '':
                buffer_line = line
            else:
                print lino, 'repaired line'
                r.write("\n\n%s %s\n%s" % (lino, 'repaired line', line))
                cols_now = len(string.split(buffer_line + line, cd))
                if cols_now == len_cols:
                    o.write(buffer_line + line + '\n')
                    buffer_line = ''

    print bigest_line
    print biggest_column
    
    o.close()
    r.close()

